{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "04_tensorboard_colab.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mlelarge/dataflowr/blob/master/Notebooks/04_deeper/04_tensorboard_colab.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "Ei8ToiWt1EzS",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Tensorboard\n"
      ]
    },
    {
      "metadata": {
        "id": "kd38ctpK1EzT",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Colab preparation for tensorboard"
      ]
    },
    {
      "metadata": {
        "id": "Uaq99uEntQOc",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!pip install tensorboardx --upgrade"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "8INqVf1stj64",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Small hack to launch Tensorboard on Google Colab. No need to do this when working locally or on your own cluster"
      ]
    },
    {
      "metadata": {
        "id": "VntggUMDtkBx",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!wget https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip\n",
        "!unzip ngrok-stable-linux-amd64.zip\n",
        "\n",
        "\n",
        "LOG_DIR = './logs/'  # \n",
        "\n",
        "\n",
        "get_ipython().system_raw(\n",
        "    'tensorboard --logdir {} --host 0.0.0.0 --port 6006 &'\n",
        "    .format(LOG_DIR)\n",
        ")\n",
        "\n",
        "import time\n",
        "time.sleep(2)\n",
        "\n",
        "get_ipython().system_raw('./ngrok http 6006 &')\n",
        "\n",
        "time.sleep(2)\n",
        "\n",
        "!curl -s http://localhost:4040/api/tunnels | python3 -c \\\n",
        "    \"import sys, json; print(json.load(sys.stdin)['tunnels'][0]['public_url'])\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "GcUIXFW8thlB",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Tensorboard examples\n"
      ]
    },
    {
      "metadata": {
        "id": "dGa3kAa8ubwL",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from tensorboardX import SummaryWriter"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Xn5DElvAugnL",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "jEKc30JO1EzU",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import torchvision.utils as vutils\n",
        "import numpy as np\n",
        "import torchvision.models as models\n",
        "from torchvision import datasets"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "IoNBMHzsuKo3",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Displaying stuff during training"
      ]
    },
    {
      "metadata": {
        "id": "XRIPb5_yugFj",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "resnet18 = models.resnet18(False)\n",
        "sample_rate = 44100\n",
        "freqs = [262, 294, 330, 349, 392, 440, 440, 440, 440, 440, 440]\n",
        "\n",
        "writer = SummaryWriter(log_dir=f'{LOG_DIR}/example1')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "yNPpj4A4uyyi",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "for n_iter in range(100):\n",
        "\n",
        "    dummy_s1 = torch.rand(1)\n",
        "    dummy_s2 = torch.rand(1)\n",
        "    # data grouping by `slash`\n",
        "    writer.add_scalar('data/scalar1', dummy_s1[0], n_iter)\n",
        "    writer.add_scalar('data/scalar2', dummy_s2[0], n_iter)\n",
        "\n",
        "    writer.add_scalars('data/scalar_group', {'xsinx': n_iter * np.sin(n_iter),\n",
        "                                             'xcosx': n_iter * np.cos(n_iter),\n",
        "                                             'arctanx': np.arctan(n_iter)}, n_iter)\n",
        "\n",
        "    dummy_img = torch.rand(32, 3, 64, 64)  # output from network\n",
        "    if n_iter % 10 == 0:\n",
        "        x = vutils.make_grid(dummy_img, normalize=True, scale_each=True)\n",
        "        writer.add_image('Image', x, n_iter)\n",
        "\n",
        "        dummy_audio = torch.zeros(sample_rate * 2)\n",
        "        for i in range(x.size(0)):\n",
        "            # amplitude of sound should in [-1, 1]\n",
        "            dummy_audio[i] = np.cos(freqs[n_iter // 10] * np.pi * float(i) / float(sample_rate))\n",
        "        writer.add_audio('myAudio', dummy_audio, n_iter, sample_rate=sample_rate)\n",
        "\n",
        "        writer.add_text('Text', 'text logged at step:' + str(n_iter), n_iter)\n",
        "\n",
        "        for name, param in resnet18.named_parameters():\n",
        "            writer.add_histogram(name, param.clone().cpu().data.numpy(), n_iter)\n",
        "\n",
        "        # needs tensorboard 0.4RC or later\n",
        "        writer.add_pr_curve('xoxo', np.random.randint(2, size=100), np.random.rand(100), n_iter)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "NQAGxePTup0L",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "writer.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "MSubmI2SwC4M",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Tensorboard aggregates plots from multiple experiments in the same folder. To this  effect we can create a new instance of `SummaryWriter()` and use the same names for the the plots"
      ]
    },
    {
      "metadata": {
        "id": "wtJ24wwPuvWZ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "writer = SummaryWriter(log_dir=f'{LOG_DIR}/example2')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Z_d4essqu2gB",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "for n_iter in range(100):\n",
        "\n",
        "    dummy_s1 = torch.rand(1)\n",
        "    dummy_s2 = torch.rand(1)\n",
        "    # data grouping by `slash`\n",
        "    writer.add_scalar('data/scalar1', dummy_s1[0], n_iter)\n",
        "    writer.add_scalar('data/scalar2', dummy_s2[0], n_iter)\n",
        "\n",
        "    writer.add_scalars('data/scalar_group', {'xsinx': n_iter * np.sin(n_iter),\n",
        "                                             'xcosx': n_iter * np.cos(n_iter),\n",
        "                                             'arctanx': np.arctan(n_iter)}, n_iter)\n",
        "\n",
        "    dummy_img = torch.rand(32, 3, 64, 64)  # output from network\n",
        "    if n_iter % 10 == 0:\n",
        "        x = vutils.make_grid(dummy_img, normalize=True, scale_each=True)\n",
        "        writer.add_image('Image', x, n_iter)\n",
        "\n",
        "        dummy_audio = torch.zeros(sample_rate * 2)\n",
        "        for i in range(x.size(0)):\n",
        "            # amplitude of sound should in [-1, 1]\n",
        "            dummy_audio[i] = np.cos(freqs[n_iter // 10] * np.pi * float(i) / float(sample_rate))\n",
        "        writer.add_audio('myAudio', dummy_audio, n_iter, sample_rate=sample_rate)\n",
        "\n",
        "        writer.add_text('Text', 'text logged at step:' + str(n_iter), n_iter)\n",
        "\n",
        "        for name, param in resnet18.named_parameters():\n",
        "            writer.add_histogram(name, param.clone().cpu().data.numpy(), n_iter)\n",
        "\n",
        "        # needs tensorboard 0.4RC or later\n",
        "        writer.add_pr_curve('xoxo', np.random.randint(2, size=100), np.random.rand(100), n_iter)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "sx9e_pXlCuti",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "dataset = datasets.MNIST('mnist', train=False, download=True)\n",
        "images = dataset.test_data[:100].float()\n",
        "label = dataset.test_labels[:100]\n",
        "\n",
        "features = images.view(100, 784)\n",
        "writer.add_embedding(features, metadata=label, label_img=images.unsqueeze(1))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "G1hzModKwTFB",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "writer.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "yzS-W6WSwYeQ",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Visualizing architectures"
      ]
    },
    {
      "metadata": {
        "id": "FiuQgdtcwmeG",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "#### A few simple nets"
      ]
    },
    {
      "metadata": {
        "id": "8o69Odrgx4sY",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torchvision\n",
        "\n",
        "dummy_input = (torch.zeros(1, 3),)\n",
        "\n",
        "\n",
        "class LinearInLinear(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(LinearInLinear, self).__init__()\n",
        "        self.l = nn.Linear(3, 5)\n",
        "\n",
        "    def forward(self, x):\n",
        "        return self.l(x)\n",
        "\n",
        "\n",
        "with SummaryWriter(log_dir=f'{LOG_DIR}/LinearInLinear', comment='LinearInLinear') as w:\n",
        "    w.add_graph(LinearInLinear(), dummy_input, True)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "fClvZGqh2WgW",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "class MultipleInput(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(MultipleInput, self).__init__()\n",
        "        self.Linear_1 = nn.Linear(3, 5)\n",
        "\n",
        "\n",
        "    def forward(self, x, y):\n",
        "        return self.Linear_1(x+y)\n",
        "\n",
        "with SummaryWriter(log_dir=f'{LOG_DIR}/MultipleInput',comment='MultipleInput') as w:\n",
        "    w.add_graph(MultipleInput(), (torch.zeros(1, 3), torch.zeros(1, 3)), True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "aBtzDCW13pNI",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "class MultipleOutput(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(MultipleOutput, self).__init__()\n",
        "        self.Linear_1 = nn.Linear(3, 5)\n",
        "        self.Linear_2 = nn.Linear(3, 7)\n",
        "\n",
        "    def forward(self, x):\n",
        "        return self.Linear_1(x), self.Linear_2(x)\n",
        "\n",
        "with SummaryWriter(log_dir=f'{LOG_DIR}/MultipleOutput',comment='MultipleOutput') as w:\n",
        "    w.add_graph(MultipleOutput(), dummy_input, True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "nnzPzkqv3pQ9",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "class SimpleModel(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(SimpleModel, self).__init__()\n",
        "\n",
        "    def forward(self, x):\n",
        "        return x * 2\n",
        "\n",
        "\n",
        "model = SimpleModel()\n",
        "dummy_input = (torch.zeros(1, 2, 3),)\n",
        "\n",
        "with SummaryWriter(log_dir=f'{LOG_DIR}/SimpleModel', comment='constantModel') as w:\n",
        "    w.add_graph(model, dummy_input, True)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "C-J7zt-f49_l",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "\n",
        "def conv3x3(in_planes, out_planes, stride=1):\n",
        "    \"\"\"3x3 convolution with padding\"\"\"\n",
        "    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,\n",
        "                     padding=1, bias=False)\n",
        "\n",
        "\n",
        "class BasicBlock(nn.Module):\n",
        "    expansion = 1\n",
        "\n",
        "    def __init__(self, inplanes, planes, stride=1, downsample=None):\n",
        "        super(BasicBlock, self).__init__()\n",
        "        self.conv1 = conv3x3(inplanes, planes, stride)\n",
        "        self.bn1 = nn.BatchNorm2d(planes)\n",
        "        # self.relu = nn.ReLU(inplace=True)\n",
        "        self.conv2 = conv3x3(planes, planes)\n",
        "        self.bn2 = nn.BatchNorm2d(planes)\n",
        "        self.stride = stride\n",
        "\n",
        "    def forward(self, x):\n",
        "        residual = x\n",
        "\n",
        "        out = self.conv1(x)\n",
        "        out = self.bn1(out)\n",
        "        out = F.relu(out)\n",
        "        out = self.conv2(out)\n",
        "        out = self.bn2(out)\n",
        "        out += residual\n",
        "        out = F.relu(out)\n",
        "        return out\n",
        "\n",
        "\n",
        "dummy_input = torch.rand(1, 3, 224, 224)\n",
        "\n",
        "with SummaryWriter(log_dir=f'{LOG_DIR}/BasicBlock', comment='basicblock') as w:\n",
        "    model = BasicBlock(3, 3)\n",
        "    w.add_graph(model, (dummy_input, ), verbose=True)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "F0x8U-WHwxUt",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "#### Siamese Networks"
      ]
    },
    {
      "metadata": {
        "id": "VVBb0C155rbT",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "class Net1(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(Net1, self).__init__()\n",
        "        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)\n",
        "        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)\n",
        "        self.conv2_drop = nn.Dropout2d()\n",
        "        self.fc1 = nn.Linear(320, 50)\n",
        "        self.fc2 = nn.Linear(50, 10)\n",
        "        self.bn = nn.BatchNorm2d(20)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = F.max_pool2d(self.conv1(x), 2)\n",
        "        x = F.relu(x) + F.relu(-x)\n",
        "        x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))\n",
        "        x = self.bn(x)\n",
        "        x = x.view(-1, 320)\n",
        "        x = F.relu(self.fc1(x))\n",
        "        x = F.dropout(x, training=self.training)\n",
        "        x = self.fc2(x)\n",
        "        x = F.softmax(x, dim=1)\n",
        "        return x\n",
        "\n",
        "\n",
        "class Net2(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(Net2, self).__init__()\n",
        "        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)\n",
        "        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)\n",
        "        self.conv2_drop = nn.Dropout2d()\n",
        "        self.fc1 = nn.Linear(320, 50)\n",
        "        self.fc2 = nn.Linear(50, 10)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = F.relu(F.max_pool2d(self.conv1(x), 2))\n",
        "        x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))\n",
        "        x = x.view(-1, 320)\n",
        "        x = F.relu(self.fc1(x))\n",
        "        x = F.dropout(x, training=self.training)\n",
        "        x = self.fc2(x)\n",
        "        x = F.log_softmax(x, dim=1)\n",
        "        return x\n",
        "\n",
        "\n",
        "dummy_input = torch.rand(13, 1, 28, 28)\n",
        "\n",
        "model = Net1()\n",
        "with SummaryWriter(log_dir=f'{LOG_DIR}/Net1',comment='Net1') as w:\n",
        "    w.add_graph(model, (dummy_input, ))\n",
        "    \n",
        "\n",
        "model = Net2()\n",
        "with SummaryWriter(log_dir=f'{LOG_DIR}/Net2',comment='Net2') as w:\n",
        "    w.add_graph(model, (dummy_input, ))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "5YZ-u7lV6DD0",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "class SiameseNetwork(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(SiameseNetwork, self).__init__()\n",
        "        self.cnn1 = Net1()\n",
        "\n",
        "    def forward_once(self, x):\n",
        "        output = self.cnn1(x)\n",
        "        return output\n",
        "\n",
        "    def forward(self, input1, input2):\n",
        "        output1 = self.forward_once(input1)\n",
        "        output2 = self.forward_once(input2)\n",
        "        return output1, output2\n",
        "\n",
        "model = SiameseNetwork()\n",
        "with SummaryWriter(log_dir=f'{LOG_DIR}/SiameseNetwork', comment='SiameseNetwork') as w:\n",
        "    w.add_graph(model, (dummy_input, dummy_input))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "BMC2QVHCw3NI",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "#### AlexNet"
      ]
    },
    {
      "metadata": {
        "id": "kcH-xHhm6W8E",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "dummy_input = torch.Tensor(1, 3, 224, 224)\n",
        "\n",
        "# with SummaryWriter(log_dir=LOG_DIR, comment='alexnet') as w:\n",
        "with SummaryWriter(log_dir=f'{LOG_DIR}/AlexNet', comment='alexnet') as w:\n",
        "\n",
        "    model = torchvision.models.alexnet()\n",
        "    w.add_graph(model, (dummy_input, ))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "6uupHfQbw7us",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "#### ResNet"
      ]
    },
    {
      "metadata": {
        "id": "0WtVBrnP6hMo",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "dummy_input = torch.Tensor(1, 3, 224, 224)\n",
        "\n",
        "with SummaryWriter(log_dir=f'{LOG_DIR}/ResNet18', comment='resnet18') as w:\n",
        "    model = torchvision.models.resnet18()\n",
        "    w.add_graph(model, (dummy_input, ))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "lrvmNUtcw_n-",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "#### DenseNet"
      ]
    },
    {
      "metadata": {
        "id": "NC5Ct4-H6ypz",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "dummy_input = torch.Tensor(1, 3, 224, 224)\n",
        "\n",
        "\n",
        "with SummaryWriter(log_dir=f'{LOG_DIR}/DenseNet121', comment='densenet121') as w:\n",
        "    model = torchvision.models.densenet121()\n",
        "    w.add_graph(model, (dummy_input, ))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "VwOHu346xGG4",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "#### RNNs"
      ]
    },
    {
      "metadata": {
        "id": "MbppPGZo7ghB",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "class RNN(nn.Module):\n",
        "    def __init__(self, input_size, hidden_size, output_size):\n",
        "        super(RNN, self).__init__()\n",
        "        self.hidden_size = hidden_size\n",
        "        self.i2h = nn.Linear(\n",
        "            n_categories +\n",
        "            input_size +\n",
        "            hidden_size,\n",
        "            hidden_size)\n",
        "        self.i2o = nn.Linear(\n",
        "            n_categories +\n",
        "            input_size +\n",
        "            hidden_size,\n",
        "            output_size)\n",
        "        self.o2o = nn.Linear(hidden_size + output_size, output_size)\n",
        "        self.dropout = nn.Dropout(0.1)\n",
        "        self.softmax = nn.LogSoftmax(dim=1)\n",
        "\n",
        "    def forward(self, category, input, hidden):\n",
        "        input_combined = torch.cat((category, input, hidden), 1)\n",
        "        hidden = self.i2h(input_combined)\n",
        "        output = self.i2o(input_combined)\n",
        "        output_combined = torch.cat((hidden, output), 1)\n",
        "        output = self.o2o(output_combined)\n",
        "        output = self.dropout(output)\n",
        "        output = self.softmax(output)\n",
        "        return output, hidden, input\n",
        "\n",
        "    def initHidden(self):\n",
        "        return torch.zeros(1, self.hidden_size)\n",
        "\n",
        "\n",
        "n_letters = 100\n",
        "n_hidden = 128\n",
        "n_categories = 10\n",
        "rnn = RNN(n_letters, n_hidden, n_categories)\n",
        "cat = torch.Tensor(1, n_categories)\n",
        "dummy_input = torch.Tensor(1, n_letters)\n",
        "hidden = torch.Tensor(1, n_hidden)\n",
        "\n",
        "\n",
        "out, hidden, input = rnn(cat, dummy_input, hidden)\n",
        "with SummaryWriter(log_dir=f'{LOG_DIR}/RNN', comment='RNN') as w:\n",
        "    w.add_graph(rnn, (cat, dummy_input, hidden), verbose=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "eRVqx_yc7yw_",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "lstm = torch.nn.LSTM(3, 3)  # Input dim is 3, output dim is 3\n",
        "inputs = [torch.randn(1, 3) for _ in range(5)]  # make a sequence of length 5\n",
        "\n",
        "# initialize the hidden state.\n",
        "hidden = (torch.randn(1, 1, 3),\n",
        "          torch.randn(1, 1, 3))\n",
        "for i in inputs:\n",
        "    out, hidden = lstm(i.view(1, 1, -1), hidden)\n",
        "\n",
        "with SummaryWriter(log_dir=f'{LOG_DIR}/LSTM', comment='lstm') as w:\n",
        "    w.add_graph(lstm, (torch.randn(1, 3).view(1, 1, -1), hidden), verbose=True)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "pMZ1VPu1xSVk",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Embedding visualization for a two class problem"
      ]
    },
    {
      "metadata": {
        "id": "RbSrRHrh9wJu",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import os\n",
        "from tensorboardX import SummaryWriter\n",
        "from torch.utils.data import TensorDataset, DataLoader\n",
        "\n",
        "\n",
        "class TwoClassNet(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(TwoClassNet, self).__init__()\n",
        "        self.cn1 = nn.Conv2d(in_channels=1, out_channels=64, kernel_size=3)\n",
        "        self.cn2 = nn.Conv2d(in_channels=64, out_channels=32, kernel_size=3)\n",
        "        self.fc1 = nn.Linear(in_features=128, out_features=2)\n",
        "\n",
        "    def forward(self, i):\n",
        "        i = self.cn1(i)\n",
        "        i = F.relu(i)\n",
        "        i = F.max_pool2d(i, 2)\n",
        "        i = self.cn2(i)\n",
        "        i = F.relu(i)\n",
        "        i = F.max_pool2d(i, 2)\n",
        "        i = i.view(len(i), -1)\n",
        "        i = self.fc1(i)\n",
        "        i = F.log_softmax(i, dim=1)\n",
        "        return i\n",
        "\n",
        "# get some random data around value\n",
        "def get_data(value, shape):\n",
        "    data = torch.ones(shape) * value\n",
        "    # add some noise\n",
        "    data += torch.randn(shape)**2\n",
        "    return data\n",
        "\n",
        "\n",
        "# dataset\n",
        "# cat some data with different values\n",
        "data = torch.cat(\n",
        "    (get_data(\n",
        "        0, (100, 1, 14, 14)), get_data(\n",
        "            0.5, (100, 1, 14, 14))), 0)\n",
        "# labels\n",
        "labels = torch.cat((torch.zeros(100), torch.ones(100)), 0)\n",
        "# generator\n",
        "gen = DataLoader(TensorDataset(data, labels), batch_size=25, shuffle=True)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Ah5uh_wdxhQY",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# network\n",
        "model = TwoClassNet()\n",
        "#loss and optim\n",
        "loss = nn.NLLLoss()\n",
        "optimizer = torch.optim.Adam(params=model.parameters())\n",
        "# settings for train and log\n",
        "num_epochs = 20\n",
        "embedding_log = 5\n",
        "writer = SummaryWriter(log_dir=f'{LOG_DIR}/embedding_demo', comment='mnist_embedding_training')\n",
        "\n",
        "# TRAIN\n",
        "for epoch in range(num_epochs):\n",
        "    for j, sample in enumerate(gen):\n",
        "        n_iter = (epoch * len(gen)) + j\n",
        "        # reset grad\n",
        "        model.zero_grad()\n",
        "        optimizer.zero_grad()\n",
        "        # get batch data\n",
        "        data_batch = sample[0].requires_grad_().float()\n",
        "        label_batch = sample[1].requires_grad_().long()\n",
        "        # FORWARD\n",
        "        out = model(data_batch)\n",
        "        loss_value = loss(out, label_batch)\n",
        "        # BACKWARD\n",
        "        loss_value.backward()\n",
        "        optimizer.step()\n",
        "        # LOGGING\n",
        "        writer.add_scalar('loss', loss_value.data.item(), n_iter)\n",
        "\n",
        "        if j % embedding_log == 0:\n",
        "            print(\"loss_value:{}\".format(loss_value.data.item()))\n",
        "            # we need 3 dimension for tensor to visualize it!\n",
        "            out = torch.cat((out.data, torch.ones(len(out), 1)), 1)\n",
        "            writer.add_embedding(\n",
        "                out,\n",
        "                metadata=label_batch.data,\n",
        "                label_img=data_batch.data,\n",
        "                global_step=n_iter)\n",
        "\n",
        "writer.close()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "zVkpy4fm_cw8",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Matplotlib"
      ]
    },
    {
      "metadata": {
        "id": "17PQuokI_gNH",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "plt.switch_backend('agg')\n",
        "\n",
        "fig = plt.figure()\n",
        "\n",
        "c1 = plt.Circle((0.2, 0.5), 0.2, color='r')\n",
        "c2 = plt.Circle((0.8, 0.5), 0.2, color='r')\n",
        "\n",
        "ax = plt.gca()\n",
        "ax.add_patch(c1)\n",
        "ax.add_patch(c2)\n",
        "plt.axis('scaled')\n",
        "\n",
        "\n",
        "from tensorboardX import SummaryWriter\n",
        "writer = SummaryWriter(log_dir=f'{LOG_DIR}/matplotlib')\n",
        "writer.add_figure('matplotlib', fig)\n",
        "writer.close()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "z2YUU_kt_xbl",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### nvidia-smi"
      ]
    },
    {
      "metadata": {
        "id": "-q0-5jHG_1Wh",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "try:\n",
        "    import nvidia_smi\n",
        "    nvidia_smi.nvmlInit()\n",
        "    handle = nvidia_smi.nvmlDeviceGetHandleByIndex(0)  # gpu0\n",
        "except ImportError:\n",
        "    print('This demo needs nvidia-ml-py or nvidia-ml-py3')\n",
        "    exit()\n",
        "    \n",
        "with SummaryWriter(log_dir=f'{LOG_DIR}/nvidia') as writer:\n",
        "    x = []\n",
        "    for n_iter in range(50):\n",
        "        x.append(torch.Tensor(1000, 1000).cuda())\n",
        "        res = nvidia_smi.nvmlDeviceGetUtilizationRates(handle)\n",
        "        writer.add_scalar('nv/gpu', res.gpu, n_iter)\n",
        "        res = nvidia_smi.nvmlDeviceGetMemoryInfo(handle)\n",
        "        writer.add_scalar('nv/gpu_mem', res.used, n_iter)\n",
        "        time.sleep(0.1)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}